'''
Generates a JobDef from an XML Settings file.
'''
import os
from xml.dom.minidom import parse
from string import Template
#======================================================================
# EXAMPLE XML FILE
#======================================================================
"""
<!-- Settings file for Dexen generator -->
<dexen_jobdef type="houdini_ea" file="models.hipnc" >
    <population initial_size="100" max_births="10000" />
    <devTask label="Development" in_path="/obj/develop/genotype" out_path="/obj/develop/phenotype" input_size="10"/>
    <evalTasks label="Eval Energy" in_path="/obj/evaluate_energy/phenotype" out_path="/obj/evaluate_energy/score" input_size="10">
        <score name="hvac" type="MIN"/>
        <score name="pv" type="MAX"/>
    </evalTasks>
    <evalTasks label="Eval Light" in_path="/obj/evaluate_daylight/phenotype" out_path="/obj/evaluate_daylight/score" input_size="10">
        <score name="daylight" type="MIN"/>
    </evalTasks>
    <feedbackTask label="Feedback" input_size="20" fitness="pareto_ranking" 
                  num_deaths="4" num_births="4" sel_deaths="worst" sel_births="best" 
                  mutation_prob="0.01" />
    <genotype>
        <geneSet from_pos="1" to_pos="5" type="float_range" min="0" max="1" />
        <geneSet from_pos="6" to_pos="22" type="int_range" min="5" max="18" />
        <geneSet from_pos="23" to_pos="28" type="int_choice">1,3,5,7,12,44,56</geneSet>
        <geneSet from_pos="29" to_pos="30" type="string_choice" >concrete,glass,brick,timber</geneSet>
    </genotype>
</dexen_jobdef>
"""
#======================================================================
# CLIENT CONFIG TEMPLATE
#======================================================================
# $SCORE_SETTINGS
CLIENT_CONFIG_TEMPLATE = """ 
scores = {
$SCORE_SETTINGS
}
"""
#======================================================================
# COMMON TEMPLATE
#======================================================================
COMMON_TEMPLATE = """ 
import os
import sys

#Current working directory
HOU_FOLDER_PATH = os.getcwd()

#Function to set up houdini correctly#\
#http://www.sidefx.com/docs/houdini11.0/hom/commandline
def enable_hou_module():
    '''Set up the environment so that "import hou" works.'''
    # Importing hou will load in Houdini's libraries and initialize Houdini.
    # In turn, Houdini will load any HDK extensions written in C++.  These
    # extensions need to link against Houdini's libraries, so we need to
    # make sure that the symbols from Houdini's libraries are visible to
    # other libraries that Houdini loads.  So, we adjust Python's dlopen
    # flags before importing hou.
    if hasattr(sys, "setdlopenflags"):
        old_dlopen_flags = sys.getdlopenflags()
        import DLFCN
        sys.setdlopenflags(old_dlopen_flags | DLFCN.RTLD_GLOBAL)

    try:
        import hou
    except ImportError:
        # Add $HFS/houdini/python2.5libs to sys.path so Python can find the
        # hou module.
        sys.path.append( os.environ['HFS'] + "/houdini/python%d.%dlibs" % sys.version_info[:2])
        import hou
    finally:
        if hasattr(sys, "setdlopenflags"):
            sys.setdlopenflags(old_dlopen_flags)

#Function to create empty folder
def create_folder(folder_path):
    if not os.path.exists(folder_path):
        os.mkdir(folder_path)
    else:
        for the_file in os.listdir(folder_path):
            file_path = os.path.join(folder_path, the_file)
            try:
                if os.path.isfile(file_path):
                    os.unlink(file_path)
            except Exception, e:
                print e
"""
#======================================================================
# DEV TASK TEMPLATE
#======================================================================
# $CLASS_NAME
# $HIP_FILE_NAME
# $IN_PATH
# $OUT_PATH
DEV_TASK_TEMPLATE = """ 
'''
Development Task 
The development task generates a phenotype using the parameters in the genotype.
'''
import os
from dexen import BaseSlaveTask
from individual import Individual
from slave_tasks import common
common.enable_hou_module()
import hou

#Function to run Houdini
def run_houdini_dev(ind, hip_file_name, in_path, out_path):
    #open the hipnc file
    hou_file_path = os.path.join(common.HOU_FOLDER_PATH, hip_file_name)
    hou.hipFile.load(hou_file_path)
    #set the parameters using the individual's genes
    genotype_node = hou.node(in_path)
    if not genotype_node:
        raise Exception("ERROR: Houdini node " + in_path + " does not exist.")
    for i in range(len(ind.genotype)):
        genotype_node.setParms({"gene_"+str(i+1):ind.genotype[i].value})
    #save phenotype to file
    phen_dir_path = os.path.join(common.HOU_FOLDER_PATH, "temp")
    common.create_folder(phen_dir_path)
    phen_file_path = os.path.join(phen_dir_path, "temp.bgeo")
    phenotype_node = hou.node(out_path)
    if not phenotype_node:
        raise Exception("ERROR: Houdini node " + out_path + " does not exist.")
    phenotype_node.setParms(dict([["file",phen_file_path]]))
    phenotype_node.cook()
    #read the phenotype file
    f = open(phen_file_path, "rb")
    phenotype = f.read()
    f.close()
    #return the phenotype (as a string)
    return phenotype
    
#Slave task for development
class $CLASS_NAME(BaseSlaveTask):
    def __init__(self):
        pass
    def __execute__(self, master_state, inds):
        #process individuals
        for ind in inds:
            assert isinstance(ind, Individual)
            ind.phenotype = run_houdini_dev(ind, "$HIP_FILE_NAME", "$IN_PATH", "$OUT_PATH")
            ind.archive_data("genotype.txt", " ".join(map(lambda x: str(x.value), ind.genotype)))
            ind.archive_data("phenotype.bgeo", ind.phenotype)
        #return the inds with the phenotype added
        return inds
"""
#======================================================================
# EVAL TASK TEMPLATE
#======================================================================
# nothing needs to be changed
EVAL_TASK_TEMPLATE = """
'''
Evaluation Tasks
The evaluation tasks generate scores for phenotypes.
'''
import os
from dexen import BaseSlaveTask
from individual import Individual
from slave_tasks import common
common.enable_hou_module()
import hou

#Function to run Houdini
def run_houdini_eval(ind, hip_file_name, in_path, out_path, names):
    #write the phenotype to a temporary file
    phen_dir_path = os.path.join(common.HOU_FOLDER_PATH, "temp")
    common.create_folder(phen_dir_path)
    phen_file_path = os.path.join(phen_dir_path, "temp.bgeo")
    f = open(phen_file_path, "wb")
    f.write(ind.phenotype)
    f.close()
    #open the hipnc file
    hou_file_path = os.path.join(common.HOU_FOLDER_PATH, hip_file_name)
    hou.hipFile.load(hou_file_path)
    #load the geometry into the phenotype node
    phenotype_node = hou.node(in_path)
    if not phenotype_node:
        raise Exception("ERROR: Houdini node " + in_paths[i] + " does not exist.")
    phenotype_node.setParms(dict([["file",phen_file_path]]))
    #cook the score node
    score_node = hou.node(out_path)
    if not score_node:
        raise Exception("ERROR: Houdini node " + out_paths[i] + " does not exist.")
    score_node.cook()
    #get all the scores
    scores = []
    for i in range(len(names)):
        score = score_node.geometry().attribValue(names[i])
        scores.append(score)
    #return the scores
    return scores
"""
# $CLASS_NAME
# $HIP_FILE_NAME
# $IN_PATH
# $OUT_PATH
# $NAMES
EVAL_TASK_CLASS_TEMPLATE = """
#Slave task for evaluation A
class $CLASS_NAME(BaseSlaveTask):
    def __init__(self):
        pass
    def __execute__(self, master_state, inds):
        #set the attrib names and paths
        names = $NAMES
        #process individuals
        for ind in inds:
            assert isinstance(ind, Individual)
            scores = run_houdini_eval(ind,"$HIP_FILE_NAME","$IN_PATH","$OUT_PATH",names)
            data_text = "# Data generated using Houdini file $HIP_FILE_NAME \\n\\n\\n" 
            for i in range(len(scores)):
                ind.__setattr__(names[i], scores[i])
                data_text += names[i] + " = " + str(scores[i]) + "\\n\\n"
            ind.archive_data("$CLASS_NAME.txt", data_text)
            if ind.is_fully_evaluated():
                ind.phenotype = "stripped"
        #return the inds with the scores added 
        return inds
"""
#======================================================================
# FEEDBACK TASK TEMPLATE
#======================================================================
# $CLASS_NAME
# $REGISTER_SCORES
# $FITNESS - not used at the moment
# $NUM_BIRTHS
# $NUM_DEATHS
# $SEL_BIRTHS
# $SEL_DEATHS
# $MUTATE_PROB - in constants file so not used here
FEEDBACK_TASK_TEMPLATE = """
'''
Evaluation Tasks
The feedback task performs selection and reproduction.
'''
import random
from dexen import BaseSlaveTask
from dexen.libs import ranking
from dexen.libs.ranking import ParetoRanking
import constants

#Function to select best individuals
def best(ranked, num):
    return [x.ind for x in ranked[:num]]
    
#Function to select worst individuals
def worst(ranked, num):
    return [x.ind for x in ranked[len(ranked)-num:]]
    
#Function to perform roulette wheel selection
#http://www.cse.unr.edu/~banerjee/selection.htm
def roulette(ranked, num, popsel):
    inds = []
    while (len(inds) < num):
        partsum = 0
        select = 0
        randnum = random.random()
        for i in range(0, len(popsel)):
            partsum += popsel[i]
            if partsum >= randnum:
                select = i
                break
        inds.append(ranked[select].ind)
    return inds
    
#Function to perform roulette wheel selection - of best individuals
def roulette_best(ranked, num):
    popsel = [x.rank_inv_norm for x in ranked]
    return roulette(ranked, num, popsel)
    
#Function to perform roulette wheel selection - of worst individuals
def roulette_worst(ranked, num):
    popsel = [x.rank_norm for x in ranked]
    return roulette(ranked, num, popsel)
    
#Function to invert and normalise the rank
def normalize_rank(ranked):
    max_rank = ranked[-1].rank
    for i in ranked:
        i.rank_inv = max_rank - i.rank + 1
    total_rank = sum([x.rank for x in ranked])
    total_inv_rank = sum([x.rank_inv for x in ranked])
    for i in ranked:
        i.rank_norm = float(i.rank)/float(total_rank)
        i.rank_inv_norm = float(i.rank_inv)/float(total_inv_rank)
    
#Function to create new individuals
def birth(ranked, num_births):
    #make sure we have an even number of births
    if num_births % 2 != 0:
        if num_births == 1:
            num_births == 2
        else:
            num_births = num_births - 1
    #select parents
    parents = $SEL_BIRTHS(ranked, num_births)
    assert len(parents) % 2 == 0
    random.shuffle(parents)
    children = []
    for i in range(0, len(parents), 2):
        child1, child2 = parents[i].reproduce(parents[i+1])
        child1.archive_data("parents.txt", str(parents[i]._get_id()) + " " + str(parents[i+1]._get_id()))
        child2.archive_data("parents.txt", str(parents[i]._get_id()) + " " + str(parents[i+1]._get_id()))
        children.append(child1)
        children.append(child2)
    return children

#Function to kill individuals
def death(ranked, num_deaths):
    deceased = $SEL_DEATHS(ranked, num_deaths)
    for ind in deceased:
        ind.kill()

#Slave task for feedback
class $CLASS_NAME(BaseSlaveTask):
    def __init__(self):
        pass
    def __execute__(self, master_state, inds):
        #strip all phenotypes that are no longer needed
        for ind in inds:
            ind.phenotype = "stripped"
        #rank individuals
        p_ranking = ParetoRanking(inds)
$REGISTER_SCORES
        ranked = p_ranking.rank() # returns list of RankResult sorted by rank
        normalize_rank(ranked)
        #create children using high rank individuals
        children = birth(ranked, $NUM_BIRTHS)
        #kill the low rank individuals
        death(ranked, $NUM_DEATHS)
        #return all the inds (inds + children)
        return inds + children
"""
#======================================================================
# MASTER TASK TEMPLATE
#======================================================================
# $START_TASKS
MASTER_TASK_TEMPLATE = """
'''
Master task
Registers slave tasks.
'''
import traceback
import dexen
import filters
from individual import Individual
from slave_tasks.dev_task import *
from slave_tasks.eval_tasks import *
from slave_tasks.feedback_task import *

# Master task
class MasterTask(dexen.BaseMasterTask):
    def __init__(self, master_env):
        #register slave tasks
        try:
            assert isinstance(master_env, dexen.MasterEnv)
$START_TASKS
        except:
            traceback.print_tb(sys.exc_info()[2])
        #initialise population
        for _ in range(constants.INITIAL_POP_SIZE):
            ind = Individual()
            master_env.set_ind(ind)
        
    def __execute__(self, master_env):
        assert isinstance(master_env, dexen.MasterEnv)
        pop_size = master_env.num_inds()
        if pop_size >= constants.MAX_BIRTHS:
            master_env.stop_master()

"""
#======================================================================
# FILTERS TEMPLATE
#======================================================================
# $FILTERS_FUNCTIONS
FILTERS_TEMPLATE = """
'''
Filter functions
Filter funtions are used to filter out the valid individuals.
'''
from individual import Individual

$FILTERS_FUNCTIONS
"""
#======================================================================
# INDIVIDUAL TEMPLATE
#======================================================================
# $GENOTYPE
# $INITIALISATION
# $FULLY_EVALUATED
INDIVIDUAL_TEMPLATE = """
'''
Individual
The individual represents a solutionn to the problem being optimised.
'''
import random
from dexen import BaseIndividual
import constants
import copy

ALIVE = "Alive"
DEAD = "Dead"

class Gene(object):
    def __init__(self):
        self.value = None
    def randomize():
        raise NonImplementedError()
        
class GeneFloatRange(Gene):
    def __init__(self, min, max):
        self.min = min
        self.max = max
    def randomize(self): 
        self.value = random.uniform(self.min,self.max)
        
class GeneIntRange(Gene):
    def __init__(self, min, max):
        self.min = min
        self.max = max
    def randomize(self): 
        self.value = random.randint(self.min,self.max)
        
class GeneIntChoice(Gene):
    def __init__(self, choices):
        self.choices = choices
    def randomize(self):
        self.value = random.choice(self.choices)

class GeneStringChoice(Gene):
    def __init__(self, choices):
        self.choices = choices
    def randomize(self):
        self.value = random.choice(self.choices)
        
$GENOTYPE
        
class Individual(BaseIndividual):
    #constructor
    def __init__(self):
        BaseIndividual.__init__(self)
        self.state = ALIVE
        self.genotype = copy.deepcopy(GENOTYPE)
        for gene in self.genotype:
            gene.randomize()
        self.phenotype = None
        
$INITIALISATION
        
    #crossover method for creating children
    def _crossover(self, ind):
        child1 = Individual()
        child2 = Individual()
        child1.genotype = []
        child2.genotype = []
        
        if len(GENOTYPE) == 2:
            z = 1
        else:
            z = random.randint(1, len(GENOTYPE)-2)
        
        for i in range(z):
            child1.genotype.append(self.genotype[i])
            child2.genotype.append(ind.genotype[i])
            
        for i in range(z, len(GENOTYPE)):
            child1.genotype.append(ind.genotype[i])
            child2.genotype.append(self.genotype[i])
        
        return child1, child2
        
    #mutation method for creating children
    def _mutate(self):
        for gene in self.genotype:
            if random.random() < constants.MUTATION_PROB:
                gene.randomize()
    
    #reproduction method for creating children
    def reproduce(self, ind):
        child1, child2 = self._crossover(ind)
        child1._mutate()
        child2._mutate()
        return  child1, child2
    
    #kill method for killing an individual
    def kill(self):
        self.state = DEAD
    
    #alive method
    def is_alive(self):
        return self.state == ALIVE 
    
    #is fully evaluated method
    def is_fully_evaluated(self):
$FULLY_EVALUATED
            return True
        return False
"""

#======================================================================
# CONSTANTS TEMPLATE
#======================================================================
# $CONSTANTS
CONSTANTS_TEMPLATE = """
'''
Evolutionary constants
'''

$CONSTANTS
"""

#======================================================================
# TEST TEMPLATE
#======================================================================
# $DEV_TASK_CLASS_NAME
# $EVAL_TASKS
# $FEEDBACK_TASK_CLASS_NAME
TEST_TEMPLATE = """
'''
Testing the jobdef
'''
from individual import Individual
from slave_tasks.dev_task import *
from slave_tasks.eval_tasks import *
from slave_tasks.feedback_task import *
import constants

pop = []
for i in range(constants.INITIAL_POP_SIZE):
    pop.append(Individual())
print "pop size = " + str(len(pop))

dev_task = $DEV_TASK_CLASS_NAME()
pop = dev_task.__execute__(None, pop)
print "Done Development!"

$EVAL_TASKS

feedback_task = $FEEDBACK_TASK_CLASS_NAME()
pop = feedback_task.__execute__(None, pop)
print "Done Feedback!"
"""
#======================================================================
# XML SETTINGS CLASSES
#======================================================================
class XmlSettings(object):
    def __init__(self):
        self.population = None
        self.dev_task = None
        self.feedback_task = None
        self.eval_tasks = []
        self.genotype = []
class PopulationSettings(object):
    def __init__(self,initial_size,max_births):
        self.initial_size = str(initial_size)
        self.max_births = str(max_births)
#Tasks
class DevTaskSettings(object):
    def __init__(self,label,input_size,file_name,in_path,out_path):
        self.class_name = str(label).replace('_',' ').replace('-',' ').title().replace(' ','')
        self.filter_func_name = str(label).replace(' ','_').replace('-','_').lower()
        #save args
        self.label = str(label)
        self.input_size = str(input_size)
        self.file_name = str(file_name)
        self.in_path = str(in_path)
        self.out_path = str(out_path)
        
class EvalTaskSettings(object):
    def __init__(self,label,input_size,file_name,in_path,out_path,scores_list):
        self.class_name = str(label).replace('_',' ').replace('-',' ').title().replace(' ','')
        self.filter_func_name = str(label).replace(' ','_').replace('-','_').lower()
        #save args
        self.label = str(label)
        self.input_size = str(input_size)
        self.file_name = str(file_name)
        self.in_path = str(in_path)
        self.out_path = str(out_path)
        self.scores_list = scores_list # a list of ScoreSettings
        
class ScoreSettings(object):
    def __init__(self,prefix,name,type):
        #save args
        self.name = str(name)
        self.type = str(type)
            
class FeedbackTaskSettings(object):
    def __init__(self,label,input_size,fitness,
        num_births,num_deaths,sel_births,sel_deaths,mutation_prob):
        self.class_name = str(label).replace('_',' ').replace('-',' ').title().replace(' ','')
        self.filter_func_name = str(label).replace(' ','_').replace('-','_').lower()
        #save args
        self.label = str(label)
        self.input_size = str(input_size)
        self.fitness = str(fitness)
        self.num_births = str(num_births)
        self.num_deaths = str(num_deaths)
        self.sel_births = str(sel_births)
        self.sel_deaths = str(sel_deaths)
        self.mutation_prob = str(mutation_prob)

#Gene sets
class GeneSet(object):
    def __init__(self,from_pos,to_pos):
        self.from_pos = str(from_pos)
        self.to_pos = str(to_pos)
class GeneSetRange(GeneSet):
    def __init__(self,from_pos,to_pos,min,max):
        super(GeneSetRange,self).__init__(from_pos,to_pos)
        self.min = str(min)
        self.max = str(max)
class GeneSetFloatRange(GeneSetRange):
    def __init__(self,from_pos,to_pos,min,max):
        super(GeneSetFloatRange,self).__init__(from_pos,to_pos,min,max)
class GeneSetIntRange(GeneSetRange):
    def __init__(self,from_pos,to_pos,min,max):
        super(GeneSetIntRange,self).__init__(from_pos,to_pos,min,max)
class GeneSetChoice(GeneSet):
    def __init__(self,from_pos,to_pos,choices):
        super(GeneSetChoice,self).__init__(from_pos,to_pos)
        self.choices = map(str, choices)
class GeneSetIntChoice(GeneSetChoice):
    def __init__(self,from_pos,to_pos,choices):
        super(GeneSetIntChoice,self).__init__(from_pos,to_pos,choices)
class GeneSetStringChoice(GeneSetChoice):
    def __init__(self,from_pos,to_pos,choices):
        super(GeneSetStringChoice,self).__init__(from_pos,to_pos,choices)
#======================================================================
# XML GENERATOR
#======================================================================
class HoudiniGenerator(object):
    def __init__(self, settings_file_path):
        self.settings_file_path = settings_file_path
        self.jobdef_dir_path = os.path.dirname(settings_file_path)
        #In the jobdef root folder
        self.client_config_file_name = "client_config.py"
        self.master_task_file_name = "master_task.py"
        self.filters_file_name = "filters.py"
        self.individual_file_name = "individual.py"
        self.constants_file_name = "constants.py"
        self.test_file_name = "test.py"
        #In the slave_tasks folder
        self.slave_tasks_dir_name = "slave_tasks"
        self.dev_task_file_name = "dev_task.py"
        self.eval_task_file_name = "eval_tasks.py"
        self.feedback_task_file_name = "feedback_task.py"
        self.common_file_name = "common.py"
        # Paths
        self.client_config_file_path = os.path.join(
            self.jobdef_dir_path, self.client_config_file_name)        
        self.master_task_file_path = os.path.join(
            self.jobdef_dir_path, self.master_task_file_name)
        self.filters_file_path = os.path.join(
            self.jobdef_dir_path, self.filters_file_name)
        self.individual_file_path = os.path.join(
            self.jobdef_dir_path, self.individual_file_name)
        self.constants_file_path = os.path.join(
            self.jobdef_dir_path, self.constants_file_name)
        self.slave_tasks_dir_path = os.path.join(
            self.jobdef_dir_path, self.slave_tasks_dir_name)
        self.test_file_path = os.path.join(
            self.jobdef_dir_path, self.test_file_name)
        self.dev_task_file_path = os.path.join(
            self.slave_tasks_dir_path, self.dev_task_file_name)
        self.eval_task_file_path = os.path.join(
            self.slave_tasks_dir_path, self.eval_task_file_name)
        self.feedback_task_file_path = os.path.join(
            self.slave_tasks_dir_path, self.feedback_task_file_name)
        self.common_file_path = os.path.join(
            self.slave_tasks_dir_path, self.common_file_name)
            
    def _check_attributes(self, element, attrib_list):
        for attrib in attrib_list:
            if not element.hasAttribute(attrib):
                raise Exception("ERROR in settings file. The '"+element.tagName+"' element must have an '"+attrib+"' attribute.")
            
    def _read_xml_settings(self):
        dom = parse(self.settings_file_path)
        xml_settings = XmlSettings()
        #===========================================================================
        populationElems = dom.getElementsByTagName("population")
        devTaskElems = dom.getElementsByTagName("devTask")
        evalTaskElems = dom.getElementsByTagName("evalTask")
        feedbackTaskElems = dom.getElementsByTagName("feedbackTask")
        scoreElems = dom.getElementsByTagName("score")
        geneSetElems = dom.getElementsByTagName("geneSet")
        #Check that the right elements exist
        if len(populationElems) != 1:
            raise Exception("ERROR in settings file. There must be exactly one 'population' element.")
        if len(devTaskElems) != 1:
            raise Exception("ERROR in settings file. There must be exactly one 'devTask' element.")
        if len(evalTaskElems) < 1:
            raise Exception("ERROR in settings file. There must beat least one 'evalTask' element.")
        if len(feedbackTaskElems) != 1:
            raise Exception("ERROR in settings file. There must be exactly one 'feedbackTask' element.")
        if len(scoreElems) < len(evalTaskElems):
            raise Exception("ERROR in settings file. There must be at least one 'score' elemnt for each 'evalTask' element.")
        if len(geneSetElems) < 1:
            raise Exception("ERROR in settings file. There must be at least one 'geneSet' element in the 'devTask' element.")
        #Check the hierarchy
        for geneSetElem in geneSetElems:
            if not geneSetElem.parentNode.isSameNode(devTaskElems[0]):
                raise Exception("ERROR in settings file. The 'geneSet' elements must be in the 'devTask' element.")
        for scoreElem in scoreElems:
            if scoreElem.parentNode.tagName != "evalTask":
                raise Exception("ERROR in settings file. The 'score' elements must be in the 'evalTask' elements.")
        #Check that labels and names are unique
        evalTaskLabels = [i.getAttribute("label") for i in evalTaskElems]
        if len(evalTaskLabels) > len(set(evalTaskLabels)):
            raise Exception("ERROR in settings file. The 'evalTask' elements must have unique 'labels'.")
        scoreNames = [i.getAttribute("name") for i in scoreElems]
        if len(scoreNames) > len(set(scoreNames)):
            raise Exception("ERROR in settings file. The 'score' elements must have unique 'names'.")
        #Check the the elements have the right attributes
        self._check_attributes(populationElems[0], ["initial_size","max_births"])
        self._check_attributes(devTaskElems[0],["label","file","in_path","out_path","input_size"])
        for evalTaskElem in evalTaskElems:
            self._check_attributes(evalTaskElem,["label","file","in_path","out_path","input_size"])
        self._check_attributes(feedbackTaskElems[0],["fitness","num_deaths","num_births","sel_deaths","sel_births","mutation_prob"])
        for geneSetElem in geneSetElems:
            self._check_attributes(geneSetElem,["from_pos","to_pos","type"])
            if geneSetElem.getAttribute("type")[-5:] == "range":
                self._check_attributes(geneSetElem,["min","max"])
        for scoreElem in scoreElems:
            self._check_attributes(scoreElem,["name","type"])
        #===========================================================================
        #<population initial_size="100" max_births="10000" genotype_length="55"/>
        elem = populationElems[0]
        population_initial_size = elem.getAttribute( "initial_size")
        population_max_births = elem.getAttribute( "max_births")
        xml_settings.population = PopulationSettings(
            population_initial_size,population_max_births)
        #===========================================================================
        #<devTask label="Development" in_path="/obj/develop/genotype" out_path="/obj/develop/phenotype" input_size="10"/>
        elem = devTaskElems[0]
        dev_task_label = elem.getAttribute( "label")
        dev_task_input_size = elem.getAttribute( "input_size")
        dev_task_file = elem.getAttribute( "file")
        dev_task_in_path = elem.getAttribute( "in_path")
        dev_task_out_path = elem.getAttribute( "out_path")
        xml_settings.dev_task = DevTaskSettings(
            dev_task_label,dev_task_input_size,dev_task_file,
            dev_task_in_path,dev_task_out_path)
        #===========================================================================
        #<feedbackTask label="Feedback" input_size="50" ranking="pareto_ranking"/>
        elem = feedbackTaskElems[0]
        feedback_task_label = elem.getAttribute( "label")
        feedback_task_input_size = elem.getAttribute( "input_size")
        feedback_task_fitness = elem.getAttribute( "fitness")
        feedback_task_num_births = elem.getAttribute( "num_births")
        feedback_task_num_deaths = elem.getAttribute( "num_deaths")
        feedback_task_sel_births = elem.getAttribute( "sel_births")
        feedback_task_sel_deaths = elem.getAttribute( "sel_deaths")
        feedback_task_mutation_prob = elem.getAttribute( "mutation_prob")
        xml_settings.feedback_task = FeedbackTaskSettings(
            feedback_task_label,feedback_task_input_size,feedback_task_fitness,
            feedback_task_num_births,feedback_task_num_deaths,
            feedback_task_sel_births,feedback_task_sel_deaths,
            feedback_task_mutation_prob)
        #===========================================================================
        #<evalTasks label="Eval Energy" in_path="/obj/evaluate_energy/phenotype" out_path="/obj/evaluate_energy/score" input_size="10">
        #    <score name="hvac" type="MIN"/>
        #    <score name="pv" type="MAX"/>
        #</evalTasks>
        eval_tasks = []
        for elem in evalTaskElems:
            eval_task_label = elem.getAttribute( "label")
            eval_task_input_size = elem.getAttribute( "input_size")
            eval_task_file = elem.getAttribute( "file")
            eval_task_in_path = elem.getAttribute( "in_path")
            eval_task_out_path = elem.getAttribute( "out_path")
            score_list = []
            for child in elem.childNodes:
                if child.nodeType == child.ELEMENT_NODE and child.localName == "score":
                    score_name = child.getAttribute( "name")
                    score_type = child.getAttribute( "type")
                    score = ScoreSettings(eval_task_label, score_name, score_type)
                    score_list.append(score)
            xml_settings.eval_tasks.append(
                EvalTaskSettings(
                    eval_task_label,eval_task_input_size,eval_task_file,
                    eval_task_in_path,eval_task_out_path,score_list))
        #===========================================================================
        #<geneSet from_pos="1" to_pos="5" type="float_range" min="0" max="1" />
        #<geneSet from_pos="6" to_pos="22" type="int_range" min="5" max="18" />
        #<geneSet from_pos="23" to_pos="28" type="int_choice">1,3,5,7,12,44,56</geneType>
        #<geneSet from_pos="29" to_pos="30" type="string_choice" >concrete,glass,brick,timber</geneType>
        gene_ranges = []
        for elem in geneSetElems:
            gene_set_from_pos = elem.getAttribute( "from_pos")
            gene_set_to_pos = elem.getAttribute( "to_pos")
            gene_set_type = elem.getAttribute( "type")
            gene_set = None
            if gene_set_type == "float_range":
                gene_set_min = elem.getAttribute( "min")
                gene_set_max = elem.getAttribute( "max")
                gene_set = GeneSetFloatRange(
                    gene_set_from_pos, gene_set_to_pos, gene_set_min, gene_set_max)
            elif gene_set_type == "int_range":
                gene_set_min = elem.getAttribute( "min")
                gene_set_max = elem.getAttribute( "max")
                gene_set = GeneSetIntRange(
                    gene_set_from_pos, gene_set_to_pos, gene_set_min, gene_set_max)
            elif gene_set_type == "int_choice":
                gene_set_data = elem.childNodes[0].data.split(',')
                gene_set = GeneSetIntChoice(
                    gene_set_from_pos, gene_set_to_pos, gene_set_data)
            elif gene_set_type == "string_choice":
                gene_set_data = elem.childNodes[0].data.split(',')
                gene_set = GeneSetStringChoice(
                    gene_set_from_pos, gene_set_to_pos, gene_set_data)
            xml_settings.genotype.append(gene_set)
        #===========================================================================
        return xml_settings
        
    #Delete python files in folder
    def _cleanup_folder(self, folder_path):
        for the_file in os.listdir(folder_path):
            if the_file.endswith(".py") or the_file.endswith(".pyc"):
                file_path = os.path.join(folder_path, the_file)
                try:
                    if os.path.isfile(file_path):
                        os.unlink(file_path)
                except Exception, e:
                    print e
                    
    #Create client config file 
    def _create_client_config(self, settings):
        s = settings
        ss = "" #$SCORE SETTINGS
        for et in s.eval_tasks:
            for sc in et.scores_list:
                ss += "    '" + sc.name + "':'" + sc.type + "',\n"
        t = Template(CLIENT_CONFIG_TEMPLATE)
        f = open(self.client_config_file_path, "w")
        f.write(t.substitute(SCORE_SETTINGS=ss))
        f.close()
        
    #Create the file with all the constants
    def _create_constants(self, settings):
        s = settings
        c = "" #$CONSTANTS
        c += "INITIAL_POP_SIZE = " + s.population.initial_size + "\n" 
        c += "MAX_BIRTHS = " + s.population.max_births + "\n"
        c += "MUTATION_PROB = " + s.feedback_task.mutation_prob + "\n"
        t = Template(CONSTANTS_TEMPLATE)
        f = open(self.constants_file_path, "w")
        f.write(t.substitute(CONSTANTS=c))
        f.close()
                    
    #Create the master task file
    def _create_master_task(self, settings):
        s = settings
        st = "" #$START_TASKS
        st += "            "
        st += "master_env.start_task(" + s.dev_task.class_name + ", "
        st += "'" + s.dev_task.label + "', " + s.dev_task.input_size + ", "
        st += "filters." + s.dev_task.filter_func_name + ")\n"
        for et in s.eval_tasks:
            st += "            "
            st += "master_env.start_task(" + et.class_name + ", "
            st += "'" + et.label + "', " + et.input_size + ", "                 #TODO: change this to use constants
            st += "filters." + et.filter_func_name + ")\n"
        st += "            "
        st += "master_env.start_task(" + s.feedback_task.class_name + ", "
        st += "'" + s.feedback_task.label + "', " + s.feedback_task.input_size + ", "
        st += "filters." + s.feedback_task.filter_func_name + ")\n"
        t = Template(MASTER_TASK_TEMPLATE)
        f = open(self.master_task_file_path, "w")
        f.write(t.substitute(START_TASKS=st))
        f.close()
        
    #Create the filters file
    def _create_filters(self, settings):
        s = settings
        ff = "" #$FILTERS_FUNCTIONS
        ff += "def " + s.dev_task.filter_func_name + "(ind): \n"
        ff += "    assert isinstance(ind, Individual) \n"
        ff += "    return ind.is_alive() and ind.genotype != None and ind.phenotype == None \n"
        for et in s.eval_tasks:
            ff += "def " + et.filter_func_name + "(ind): \n"
            ff += "    assert isinstance(ind, Individual) \n"
            ff += "    return ind.is_alive() and ind.phenotype != None "
            for sc in et.scores_list:
                ff += "and ind." + sc.name + " == None "
            ff += "\n"
        ff += "def " + s.feedback_task.filter_func_name + "(ind): \n"
        ff += "    assert isinstance(ind, Individual) \n"
        ff += "    return ind.is_alive() "
        for et in s.eval_tasks:
            for sc in et.scores_list:
                ff += "and ind." + sc.name + " != None "
        t = Template(FILTERS_TEMPLATE)
        f = open(self.filters_file_path, "w")
        f.write(t.substitute(FILTERS_FUNCTIONS=ff))
        f.close()
        
    #Create the individual file
    def _create_individual(self, settings):
        s = settings
        gt = "" #$GENOTYPE
        gt += "GENOTYPE = [] \n"
        sorted_gs = sorted(s.genotype, key=lambda g: int(g.from_pos))
        for g in sorted_gs:
            #print type(g.from_pos)
            #print g.from_pos
            if isinstance(g, GeneSetFloatRange):
                gt += "GENOTYPE += [GeneFloatRange(" + g.min + "," + g.max + ") "
                gt += "for i in range(" + g.from_pos + "," + str(int(g.to_pos) + 1) + ")]\n"
            elif isinstance(g, GeneSetIntRange):
                gt += "GENOTYPE += [GeneIntRange(" + g.min + "," + g.max + ") "
                gt += "for i in range(" + g.from_pos + "," + str(int(g.to_pos) + 1) + ")]\n"
            elif isinstance(g, GeneSetIntChoice):
                gt += "GENOTYPE += [GeneIntChoice([" + ",".join(g.choices) + "]) "
                gt += "for i in range(" + g.from_pos + "," + str(int(g.to_pos) + 1) + ")]\n"
            elif isinstance(g, GeneSetStringChoice):
                gt += "GENOTYPE += [GeneStringChoice(['" + "','".join(g.choices) + "']) "
                gt += "for i in range(" + g.from_pos + "," + str(int(g.to_pos) + 1) + ")]\n"
        ii = "" #$INITIALISATION
        for et in s.eval_tasks:
            for sc in et.scores_list:
                ii += "        "
                ii += "self." + sc.name + " = None \n"
        fe = "" #FULLY_EVALUATED
        fe += "        "
        fe += "if "
        for et in s.eval_tasks:
            for sc in et.scores_list:
                fe += "self." + sc.name + " != None and "
        fe = fe[:-4]  + ":"
        t = Template(INDIVIDUAL_TEMPLATE)
        f = open(self.individual_file_path, "w")
        f.write(t.substitute(GENOTYPE=gt, INITIALISATION=ii, FULLY_EVALUATED=fe))
        f.close()
        
    #Create the dev task file
    def _create_dev_task(self, settings):
        s = settings
        t = Template(DEV_TASK_TEMPLATE)
        f = open(self.dev_task_file_path, "w")
        f.write(t.substitute(
            CLASS_NAME = s.dev_task.class_name,
            HIP_FILE_NAME = s.dev_task.file_name,
            IN_PATH = s.dev_task.in_path,
            OUT_PATH = s.dev_task.out_path
            ))
        f.close()
        
    #Create the eval task file
    def _create_eval_task(self, settings):
        s = settings
        f = open(self.eval_task_file_path, "w")
        f.write(EVAL_TASK_TEMPLATE)
        for et in s.eval_tasks:
            names = []
            for sc in et.scores_list:
                names.append(sc.name)
            t = Template(EVAL_TASK_CLASS_TEMPLATE)
            f.write(t.substitute(
                CLASS_NAME = et.class_name, 
                HIP_FILE_NAME = et.file_name,
                IN_PATH = et.in_path,
                OUT_PATH = et.out_path,
                NAMES = str(names)))
        f.close()
        
    #Create the feedback task file
    def _create_feedback_task(self, settings):
        s = settings
        rs = ""
        for et in s.eval_tasks:
            for sc in et.scores_list:
                rs += "        "
                rs += "p_ranking.register_score('" + sc.name + "', "
                rs += "ranking." + sc.type + ") \n"
        t = Template(FEEDBACK_TASK_TEMPLATE)
        f = open(self.feedback_task_file_path, "w")
        f.write(t.substitute(
            CLASS_NAME = s.feedback_task.class_name,    
            REGISTER_SCORES = rs, 
            NUM_BIRTHS = s.feedback_task.num_births,
            NUM_DEATHS = s.feedback_task.num_deaths,
            SEL_BIRTHS = s.feedback_task.sel_births,
            SEL_DEATHS = s.feedback_task.sel_deaths,
            ))
        f.close()
        
    #Create the common task file
    def _create_common(self, settings):
        f = open(self.common_file_path, "w")
        f.write(COMMON_TEMPLATE)
        f.close()
        
    #Create the test file
    def _create_test(self, settings):
        s = settings
        tf = "" #$TEST FILE
        for et in s.eval_tasks:
            tf += "task = " + et.class_name + "() \n"
            tf += "pop = task.__execute__(None, pop) \n"
            tf += "print 'Done " + et.label + "!' \n"
        t = Template(TEST_TEMPLATE)
        f = open(self.test_file_path, "w")
        f.write(t.substitute(
            EVAL_TASKS = tf,
            DEV_TASK_CLASS_NAME = s.dev_task.class_name,
            FEEDBACK_TASK_CLASS_NAME = s.feedback_task.class_name))
        f.close()
        
    # Create the folder to store the tasks
    def _create_tasks_folder(self):
        #create folder for slave tasks
        if os.path.exists(self.slave_tasks_dir_path):
            self._cleanup_folder(self.slave_tasks_dir_path)
        else:
            os.mkdir(self.slave_tasks_dir_path)
        f = open(os.path.join(self.slave_tasks_dir_path, "__init__.py"), "w")
        f.close()
        
    #Generate the jobdef folders and files
    def generate(self):
        #delete contents of jobdef folder
        self._cleanup_folder(self.jobdef_dir_path)
        self._create_tasks_folder()
        #read xml settings
        settings = self._read_xml_settings()
        #create all the files
        self._create_client_config(settings)
        self._create_constants(settings)
        self._create_individual(settings)
        self._create_master_task(settings)
        self._create_filters(settings)
        self._create_individual(settings)
        self._create_dev_task(settings)
        self._create_eval_task(settings)
        self._create_feedback_task(settings)
        self._create_common(settings)
        self._create_test(settings)
        print "done"

def main():
    h_gen = HoudiniGenerator("d:/temp/dexen/test/settings.xml")
    h_gen.generate()
    

if __name__ == "__main__":
    main()


        